{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-05-08T06:16:38.677268Z",
     "start_time": "2025-05-08T06:16:38.673268Z"
    }
   },
   "source": [
    "import random\n",
    "from tqdm import tqdm\n",
    "from transformers import AutoTokenizer\n",
    "import json\n",
    "from datasets import load_dataset\n",
    "from tokenizers import (\n",
    "    decoders,\n",
    "    models,\n",
    "    normalizers,\n",
    "    pre_tokenizers,\n",
    "    processors,\n",
    "    trainers,\n",
    "    Tokenizer,\n",
    ")\n",
    "import os"
   ],
   "outputs": [],
   "execution_count": 23
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-08T06:16:38.719840Z",
     "start_time": "2025-05-08T06:16:38.715840Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def read_texts_from_jsonl(file_path):\n",
    "    with open(file_path, 'r', encoding='utf-8') as f:\n",
    "        for line in f:\n",
    "            data = json.loads(line)\n",
    "            yield data['text']\n",
    "\n",
    "# 测试读取数据\n",
    "data_path = 'dataset/tokenizer_train.jsonl'\n",
    "texts = read_texts_from_jsonl(data_path)\n",
    "\n",
    "# 打印前几行文本\n",
    "for i, text in enumerate(texts):\n",
    "    if i < 5:\n",
    "        print(text)\n",
    "    else:\n",
    "        break"
   ],
   "id": "5de875f789c5dbf7",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "好的。现在请你将这个文本中的所有的逗号都替换成空格。 好的，请稍等一下，现在我会将文本中的所有逗号替换为空格。处理后文本为：\"这是一个句子 目的是看看是否可以正确地从这个句子中删除关键词。\"。处理结果如何？\n",
      "帮我回答一道历史题目。清朝时期的八旗共有多少旗人？ 清朝时期八旗旗人总数约为200万人左右，其中正黄旗、正蓝旗、正白旗、正红旗的人数较多，其他旗的人数较少。\n",
      "嗯，谢谢你介绍的做法很详细，但我不喜欢吃鸡蛋，有没有其他菜做法能介绍一下？ 当然，你可以试试酸辣土豆丝这道菜。\n",
      "材料：\n",
      "土豆2个、红椒1个、青椒1个、大葱1根、醋、生抽、盐、鸡精、料酒\n",
      "做法：\n",
      "1.土豆去皮，切成丝；红椒和青椒切成细丝；大葱切段备用。\n",
      "2.热锅凉油，油热后放入土豆丝，煸炒至变软。\n",
      "3.倒入红椒、青椒和大葱段，继续煸炒至熟。\n",
      "4.加入适量的盐、鸡精、料酒和生抽，翻炒均匀。\n",
      "5.最后，加入适量的醋，翻炒均匀即可。\n",
      "小贴士：\n",
      "1. 土豆切丝时，可以放入淡盐水中泡一下，这样可以去除多余的淀粉。\n",
      "2. 煮土豆丝时，不要煮得太久，以免烂糊。\n",
      "3. 加入醋的时候，根据自己的口味多少来进行调节，一般来说，盐与醋的比例为1:1。\n",
      "4. 如果喜欢辣味可以加入一些干辣椒丝。\n",
      "希望你会喜欢这道酸辣土豆丝！\n",
      "请描述一下如何正确规划个人理财。 正确规划个人理财需要以下几个步骤：\n",
      "1.了解自己的财务状况。这包括收入、支出、资产和负债等信息。了解自己的财务状况可以帮助人们更好地制定财务计划。\n",
      "2.设定财务目标。需要考虑短期目标和长期目标，例如以年为单位设定的支出计划、购房、购车等的长期目标。\n",
      "3.制定预算计划。在了解自己的财务状况并设定财务目标后，需要制定一个预算计划。这可以帮助人们控制支出、节省开支并达到财务目标。\n",
      "4.理性投资和储蓄。人们可以投资于股票、基金、房产或其他投资渠道以实现财务目标。但在投资前需了解相关知识并进行风险评估。同时还应储蓄一定金额，以应对突发事件或为达成某些目标做准备。\n",
      "5.审时度势，合理调整。财务计划需要不断地审时度势，根据实际情况做出调整，以达到最终的财务目标。需要注意财务状况的变化、投资的收益和风险等因素。\n",
      "通过以上五个步骤，人们可以做到合理规划个人理财，掌握自己的财务命运，更好地实现自己的财务目标。\n",
      "描述一下天堂和地狱的生态系统和环境。 天堂和地狱被认为是灵性信仰中关于死后世界的两种不同概念。然而，它们的生态系统和环境都是具有类似特征的极端不同的地方。以下是我对天堂和地狱的生态系统和环境的描述。\n",
      "天堂的生态系统和环境:\n",
      "天堂被描绘为一个美丽、平静、和谐的地方，类似于一片无垢的花园。天堂的生态系统和环境的特征包括:\n",
      "1. 充满和平和爱的氛围。这是一个没有恐惧、痛苦、疾病和死亡的地方。\n",
      "2. 色彩缤纷，充满生机。这是一个绿树成荫、花团锦簇的地方，充满生机和活力。\n",
      "3. 各种生物和动物和谐共存。天使、圣人和各种动物和谐相处，生态系统中没有互相侵害或抢夺资源。\n",
      "4. 充满清新气息的空气。没有污染、烟雾或其他有害物质，空气中充满清新芬芳的气息。\n",
      "5. 物质丰富的环境。天堂中生活着满足需求和愿望的人们，他们拥有一切所需的物质资源，而且没有匮乏、浪费或不公平。\n",
      "地狱的生态系统和环境:\n",
      "地狱被描绘为阴暗、恐怖、嘈杂和可怕的地方。地狱的生态系统和环境的特征包括:\n",
      "1. 充满痛苦和折磨的氛围。这是一个充满恐惧、悔恨和痛苦的地方，全是罪恶的味道。\n",
      "2. 火焰和烈火环绕。地狱中有燃烧的火焰和烈火，许多受罚者被投入火坑中痛苦折磨。\n",
      "3. 恶魔和妖魔横行。地狱中有恶魔、妖怪等可怕的生物，它们在无休止的受苦中享受着自己的又一场比赛。\n",
      "4. 污染和恶臭的气味。地狱中到处都是恶臭和污染，没有清新的气息。\n",
      "5. 没有物质资源。地狱中生活着被惩罚的人们不可能拥有任何物质财富，地狱环境充满了无尽的贫困、饥饿和疾病。\n",
      "综上所述，天堂和地狱是两个完全不同的地方，它们的生态系统和环境反映了它们的性质，体现了人类对不同阶段的死后生命的不同想象和信仰。\n"
     ]
    }
   ],
   "execution_count": 24
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-08T06:16:38.727133Z",
     "start_time": "2025-05-08T06:16:38.722982Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# with open(data_path, 'r', encoding='utf-8') as f:\n",
    "#     total_lines = sum(1 for line in f)\n",
    "# print(total_lines)\n",
    "#\n",
    "# texts = read_texts_from_jsonl(data_path)\n",
    "# with open(data_path + \"2\", 'w', encoding='utf-8') as f:\n",
    "#     with tqdm(total=total_lines) as pbar:\n",
    "#         for text in texts:\n",
    "#             f.write(text)\n",
    "#             pbar.update(1)"
   ],
   "id": "241f9776e79f6460",
   "outputs": [],
   "execution_count": 25
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-08T06:16:40.913026Z",
     "start_time": "2025-05-08T06:16:38.748682Z"
    }
   },
   "cell_type": "code",
   "source": [
    "tokenizer = Tokenizer(models.BPE())\n",
    "tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)\n",
    "special_tokens = ['<unk>', '<s>', '</s>']\n",
    "\n",
    "trainer = trainers.BpeTrainer(\n",
    "    vocab_size=6400,\n",
    "    special_tokens=special_tokens,\n",
    "    show_process=True,\n",
    "    initial_alphabet=pre_tokenizers.ByteLevel.alphabet()\n",
    ")\n",
    "print(\"分词器初始化完成,准备训练\")"
   ],
   "id": "a2366e80b88d4ec7",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "分词器初始化完成,准备训练\n"
     ]
    }
   ],
   "execution_count": 26
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-08T06:16:40.928163Z",
     "start_time": "2025-05-08T06:16:40.924190Z"
    }
   },
   "cell_type": "code",
   "source": "texts = read_texts_from_jsonl(data_path)",
   "id": "83ca501ac0cb3fec",
   "outputs": [],
   "execution_count": 27
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-08T06:24:43.420794Z",
     "start_time": "2025-05-08T06:16:40.935320Z"
    }
   },
   "cell_type": "code",
   "source": [
    "tokenizer.train_from_iterator(texts, trainer=trainer)\n",
    "print(\"分词器训练完成\")"
   ],
   "id": "ee0efd7ac0a02dd0",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "分词器训练完成\n"
     ]
    }
   ],
   "execution_count": 28
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "在训练完毕之后，还需要设置解码器 (`tokenizer.decoder = decoders.ByteLevel()`) ，这是为了在生成文本时正确地将分词器产生的 token 序列还原回原始文本。",
   "id": "f3861a42c50efcd0"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-08T06:24:43.651302Z",
     "start_time": "2025-05-08T06:24:43.632294Z"
    }
   },
   "cell_type": "code",
   "source": [
    "tokenizer.decoder = decoders.ByteLevel()\n",
    "tokenizer_dir = 'model'\n",
    "os.makedirs(tokenizer_dir, exist_ok=True)\n",
    "tokenizer.save(os.path.join(tokenizer_dir, 'tokenizer.json'))\n",
    "tokenizer.model.save(tokenizer_dir)"
   ],
   "id": "183dd8685a307bbb",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['model\\\\vocab.json', 'model\\\\merges.txt']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 29
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-08T06:24:43.660321Z",
     "start_time": "2025-05-08T06:24:43.655306Z"
    }
   },
   "cell_type": "code",
   "source": [
    "config = {\n",
    "    \"add_bos_token\": False,\n",
    "    \"add_eos_token\": False,\n",
    "    \"add_prefix_space\": True,\n",
    "    \"added_tokens_decoder\": {\n",
    "        \"0\": {\n",
    "            \"content\": \"<unk>\",\n",
    "            \"lstrip\": False,\n",
    "            \"normalized\": False,\n",
    "            \"rstrip\": False,\n",
    "            \"single_word\": False,\n",
    "            \"special\": True\n",
    "            },\n",
    "        \"1\": {\n",
    "            \"content\": \"<s>\",\n",
    "            \"lstrip\": False,\n",
    "            \"normalized\": False,\n",
    "            \"rstrip\": False,\n",
    "            \"single_word\": False,\n",
    "            \"special\": True\n",
    "            },\n",
    "        \"2\": {\n",
    "            \"content\": \"</s>\",\n",
    "            \"lstrip\": False,\n",
    "            \"normalized\": False,\n",
    "            \"rstrip\": False,\n",
    "            \"single_word\": False,\n",
    "            \"special\": True\n",
    "            }\n",
    "    },\n",
    "    \"bos_token\": \"<s>\",\n",
    "    \"clean_up_tokenization_spaces\": False,\n",
    "    \"eos_token\": \"</s>\",\n",
    "    \"legacy\": True,\n",
    "    \"model_max_length\": 1000000000000000019884624838656,\n",
    "    \"pad_token\": None,\n",
    "    \"sp_model_kwargs\": {},\n",
    "    \"spaces_between_special_tokens\": False,\n",
    "    \"tokenizer_class\": \"PreTrainedTokenizerFast\",\n",
    "    \"unk_token\": \"<unk>\",\n",
    "    \"use_default_system_prompt\": False,\n",
    "    \"chat_template\": \"{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<s>user\\\\n' + content + '</s>\\\\n<s>assistant\\\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' + '\\\\n' }}{% endif %}{% endfor %}\"\n",
    "}"
   ],
   "id": "3c147a9bf0c9c05",
   "outputs": [],
   "execution_count": 30
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-08T06:24:43.689244Z",
     "start_time": "2025-05-08T06:24:43.683837Z"
    }
   },
   "cell_type": "code",
   "source": [
    "with open(os.path.join(tokenizer_dir, 'tokenizer_config.json'), 'w', encoding='utf-8') as confg_file:\n",
    "    json.dump(config, confg_file, ensure_ascii=False, indent=4)\n",
    "\n",
    "print(\"保存成功\")"
   ],
   "id": "6b82033dce23e0d9",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "保存成功\n"
     ]
    }
   ],
   "execution_count": 31
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "- Step 6.评估分词器",
   "id": "bfee3e602ce1e86c"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-08T06:24:43.747775Z",
     "start_time": "2025-05-08T06:24:43.712677Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir)\n",
    "# 测试一段对话\n",
    "messages = [\n",
    "    {\"role\": \"system\", \"content\": \"你是一个优秀的聊天机器人，总是给我正确的回应！\"},\n",
    "    {\"role\": \"user\", \"content\": '是椭圆形的'},\n",
    "    {\"role\": \"assistant\", \"content\": '456'},\n",
    "    {\"role\": \"user\", \"content\": '456'},\n",
    "    {\"role\": \"assistant\", \"content\": '789'}\n",
    "]\n",
    "\n",
    "# 使用模板进行文本处理\n",
    "new_prompt = tokenizer.apply_chat_template(messages, tokenize=True)\n",
    "print(new_prompt)\n"
   ],
   "id": "bd733013c3f58b95",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[223, 608, 1589, 4835, 269, 4833, 954, 4725, 270, 1170, 345, 4584, 5204, 1273, 648, 2207, 1, 2765, 201, 345, 1390, 258, 3852, 1081, 269, 2, 2251, 1, 1861, 501, 201, 22, 23, 24, 2, 2251, 1, 2765, 201, 22, 23, 24, 2, 2251, 1, 1861, 501, 201, 25, 26, 27, 2, 2251]\n"
     ]
    }
   ],
   "execution_count": 32
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-08T06:24:43.775838Z",
     "start_time": "2025-05-08T06:24:43.769766Z"
    }
   },
   "cell_type": "code",
   "source": "print(tokenizer.decode(new_prompt))",
   "id": "b0740c409365ac64",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 你是一个优秀的聊天机器人，总是给我正确的回应！<s> user\n",
      "是椭圆形的</s> \n",
      "<s> assistant\n",
      "456</s> \n",
      "<s> user\n",
      "456</s> \n",
      "<s> assistant\n",
      "789</s> \n",
      "\n"
     ]
    }
   ],
   "execution_count": 33
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-08T06:24:43.800666Z",
     "start_time": "2025-05-08T06:24:43.798651Z"
    }
   },
   "cell_type": "code",
   "source": "",
   "id": "66538e56f40e7efc",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
